%reload_ext autotime
import pandas as pd
import requests
from pprint import pprint
import json
import torch
from PIL import Image
from transformers import MllamaForConditionalGeneration, AutoProcessor
from tqdm.auto import tqdm

pd.options.plotting.backend = "plotly"
pd.set_option("display.max_columns", None)
pd.set_option("display.max_colwidth", 100)

✔️ 4.69 s (2024-12-11T14:47:55/2024-12-11T14:48:00)

2024-12-11 14:47:58.593258: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-12-11 14:47:58.611738: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2024-12-11 14:47:58.630387: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2024-12-11 14:47:58.636480: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2024-12-11 14:47:58.654940: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-12-11 14:47:59.491941: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

df = pd.read_csv("results.csv").drop_duplicates(subset="panoid")
df

✔️ 21.4 ms (2024-12-11T14:48:00/2024-12-11T14:48:00)

# Loading this model needs about 22.69GB of GPU memory
model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"

model = MllamaForConditionalGeneration.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
processor = AutoProcessor.from_pretrained(model_id)

✔️ 13.9 s (2024-12-11T14:48:00/2024-12-11T14:48:14)

The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.

Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]

for row in tqdm(df.sample(10).itertuples(index=False)):
    panoid = row.panoid
    image = Image.open(f"panoramas/{panoid}.png")
    display(image)
    messages = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": """
                    This image is a panorama from Google Street View.
                    From the image, extract the following information, in JSON format:
                    green: Percentage of the image that is green space (e.g. parks, gardens, trees, grass etc.). A number from 0-100.
                    environment: Classify the nature of the environment in this image. Built up/green/residential/shops/cafes?. A string.
                    water: If you see any streams/ponds/rivers/ocean in the image, estimate the distance to the water in meters. A number. If there is no water, return 0.
                    obscured: Proportion of view obscured by buildings (how much of total line of sight is blocked by buildings in close proximity). A number from 0-100.
                    people: the number of people you see in the image
                    cars: the number of cars you see in the image
                    bikes: the number of bikes you see in the image

                    Do not include comments in your JSON response. Only respond with the JSON object. Make sure the JSON is valid.
                """},
                {"type": "image"},
            ]
        }
    ]
    input_text = processor.apply_chat_template(messages, add_generation_prompt=True)
    inputs = processor(
        image,
        input_text,
        add_special_tokens=False,
        return_tensors="pt"
    ).to(model.device)

    for retry in range(3):
        output = model.generate(**inputs, max_new_tokens=5000)
        result = processor.decode(output[0])
        result = result[result.rindex("<|end_header_id|>") + len("<|end_header_id|>"):].strip().replace("<|eot_id|>", "")
        print("Output:")
        try:
            result = json.loads(result)
            pprint(result)
            print("\n")
            break
        except json.JSONDecodeError:
            print(f"Unable to parse: {result}")

✔️ 26.1 s (2024-12-11T14:48:14/2024-12-11T14:48:40)

0it [00:00, ?it/s]

Output:
{'bikes': 0,
 'cars': 8,
 'environment': 'residential',
 'green': 45,
 'obscured': 30,
 'people': 0,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 6,
 'environment': 'Built up',
 'green': 14,
 'obscured': 50,
 'people': 5,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 10,
 'environment': 'residential',
 'green': 20,
 'obscured': 0,
 'people': 0,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 3,
 'environment': 'residential',
 'green': 60,
 'obscured': 50,
 'people': 0,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 3,
 'environment': 'residential',
 'green': 30,
 'obscured': 60,
 'people': 0,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 2,
 'environment': 'residential',
 'green': 50,
 'obscured': 30,
 'people': 0,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 2,
 'environment': 'Built up',
 'green': 30,
 'obscured': 40,
 'people': 1,
 'water': 0}

Output:
{'bikes': 0,
 'cars': 2,
 'environment': 'residential',
 'green': 35,
 'obscured': 50,
 'people': 0,
 'water': 0}

	Index	pid	n	time	anxiousness	latitude	longitude	geometry	panoid	panolat	panolon	panoyear	panomonth
0	0	P20001	1	2023-04-25T02:51:42Z	0	-36.924795	174.738044	POINT (174.7380435 -36.92479483)	QEpZV7bnO2mBfp0weMUKEg	-36.924730	174.737826	2012.0	4.0
10	10	P20001	11	2023-04-24T00:42:25Z	0	-36.924837	174.737948	POINT (174.7379477 -36.92483659)	pUw8PmVPYZBTW26mq2DAfw	-36.924785	174.737728	2012.0	4.0
13	13	P20006	1	2023-06-03T02:45:55Z	3	-36.892203	174.740125	POINT (174.7401253 -36.89220256)	omb98QNjTPWi0uUfMsmYeg	-36.892621	174.739961	2024.0	5.0
14	15	P20009	2	2023-05-17T04:54:48Z	3	-36.923191	174.748620	POINT (174.7486203 -36.92319093)	E7B5AV3DQ1rYWDClVRo8Zg	-36.923194	174.748831	2024.0	5.0
17	19	P20009	6	2023-05-19T22:28:51Z	1	-36.923260	174.748655	POINT (174.748655 -36.92325959)	KCTcsxYCIm41XdzkYEYUQw	-36.923286	174.748840	2024.0	5.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...
1071	1176	P20540	1	2024-02-19T23:30:34Z	1	-36.973117	174.825860	POINT (174.82586 -36.97311667)	dBcQrtes_utp6TffHZ__Jw	-36.973010	174.825980	2024.0	9.0
1072	1177	P20540	2	2024-02-21T22:00:47Z	0	-36.973123	174.825923	POINT (174.8259233 -36.97312333)	tb6sBk1y284EqiD7vvjF7Q	-36.973096	174.826046	2024.0	9.0
1075	1180	P20540	5	2024-02-24T19:59:47Z	1	-36.974258	174.821245	POINT (174.821245 -36.97425833)	T-HY_IoPcli7gxo9Ejr37Q	-36.974311	174.821378	2022.0	8.0
1078	1184	P20555	1	2024-03-05T03:52:43Z	0	-36.923872	174.746452	POINT (174.7464522 -36.92387209)	QkJiPOR_Qs0v5KIDZ4vUjw	-36.923879	174.746036	2024.0	6.0
1082	1188	P20555	5	2024-03-07T21:53:02Z	2	-36.908151	174.756630	POINT (174.75663 -36.90815125)	1j-S2g_Bd98TQ7Ti8Q2LhA	-36.908054	174.756236	2024.0	6.0